collect some information when qemu-kvm exit

collect some information when qemu-kvm exit

[lidong chen]

I find qemu-kvm only output a little information when abnormally exit.
For example, if qemu-kvm exit by segmentation fault, there are no
information in /var/log/libvirt/qemu/xx.log.

so i want to solve this by collect some information when qemu-kvm exit.

my idea is register some signal handler, and print some debug
information in the signal handler function. and use atexit register a
function for process termination.

the main thread already used SIGBUS,SIGUSR2,SIGALRM,SIGIO. and  vcpu
thread used SIGBUS,SIGIPI.

the signal that should register for vcpu thread.
   SIGHUP,
   SIGINT,
   SIGQUIT,
   SIGILL,
   SIGTRAP,
   SIGABRT,
   SIGFPE,
   SIGUSR1,
   SIGPIPE,
   SIGSEGV,
   SIGUSR2,
   SIGTERM,
   SIGSTKFLT,
   SIGTSTP,
   SIGXCPU,
   SIGXFSZ,
   SIGVTALRM,
   SIGIO,
   SIGSYS,

the signal that should register for main thread.
   SIGHUP,
   SIGINT,
   SIGQUIT,
   SIGILL,
   SIGTRAP,
   SIGABRT,
   SIGFPE,
   SIGUSR1,
   SIGPIPE,
   SIGSEGV,
   SIGTERM,
   SIGSTKFLT,
   SIGTSTP,
   SIGXCPU,
   SIGXFSZ,
   SIGVTALRM,
   SIGSYS,

the simple example for this function:
--- ../../BUILD/qemu-kvm-0.12.5/vl.c 2011-05-25 04:08:00.000000000 -0400
+++ vl.c 2011-06-22 06:57:51.000000000 -0400
+static void sig_handler(int n)
+{
+         int j, nptrs;
+       #define SIZE 100
+           void *buffer[100];
+           char **strings;
+
+           nptrs = backtrace(buffer, SIZE);
+           printf("backtrace() returned %d addresses\n", nptrs);
+
+           /* The call backtrace_symbols_fd(buffer, nptrs, STDOUT_FILENO)
+              would produce similar output to the following: */
+
+           strings = backtrace_symbols(buffer, nptrs);
+           if (strings == NULL) {
+               perror("backtrace_symbols");
+               exit(EXIT_FAILURE);
+           }
+
+           for (j = 0; j < nptrs; j++)
+               printf("%s\n", strings[j]);
+
+           free(strings);
+
+       abort();
+}
+
+void bye(void) {
+        printf("That was all, folks\n");
+         int j, nptrs;
+       #define SIZE 100
+           void *buffer[100];
+           char **strings;
+
+           nptrs = backtrace(buffer, SIZE);
+           printf("backtrace() returned %d addresses\n", nptrs);
+
+           /* The call backtrace_symbols_fd(buffer, nptrs, STDOUT_FILENO)
+              would produce similar output to the following: */
+
+           strings = backtrace_symbols(buffer, nptrs);
+           if (strings == NULL) {
+               perror("backtrace_symbols");
+               exit(EXIT_FAILURE);
+           }
+
+           for (j = 0; j < nptrs; j++)
+               printf("%s\n", strings[j]);
+
+           free(strings);
+
+}
+
+
int main(int argc, char **argv, char **envp)
{
    const char *gdbstub_dev = NULL;
@@ -4954,6 +5010,17 @@

    init_clocks();

+
+    signal(SIGSEGV, sig_handler);
+
+    i = atexit(bye);
+    if (i != 0) {
+           fprintf(stderr, "cannot set exit function\n");
+           return EXIT_FAILURE;
+    }
+
+
+

--- ../../BUILD/qemu-kvm-0.12.5/qemu-kvm.c    2011-05-25
03:31:25.000000000 -0400
+++ qemu-kvm.c    2011-06-22 07:04:05.000000000 -0400
@@ -1883,6 +1883,10 @@

+
static void *ap_main_loop(void *_env)
{
    CPUState *env = _env;
@@ -1908,10 +1927,10 @@
    struct ioperm_data *data = NULL;
#endif

    current_env = env;
    env->thread_id = kvm_get_thread_id();
    sigfillset(&signals);
+    sigdelset(&signals,SIGSEGV);
    sigprocmask(SIG_BLOCK, &signals, NULL);
    kvm_create_vcpu(env, env->cpu_index);

the log in /var/log/libvirt/qemu/xx.log.

LC_ALL=C PATH=/bin:/sbin:/usr/bin:/usr/sbin HOME=/ QEMU_AUDIO_DRV=none
/usr/bin/qemu-kvm -S -M pc-0.12 -enable-kvm -m 9767 -smp
16,sockets=16,cores=1,threads=1 -name sles11-4 -uuid
52841129-6b46-fef5-6a62-11c422597206 -nodefaults -chardev
socket,id=monitor,path=/var/lib/libvirt/qemu/sles11-4.monitor,server,nowait
-mon chardev=monitor,mode=readline -no-reboot -boot dc -drive
file=/mnt/disk2/c00104598/sles11sp1.raw,if=none,id=drive-virtio-disk0,boot=on
-device virtio-blk-pci,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0
-drive file=/opt/c00104598/iso/SLES-11-SP1.iso,if=none,media=cdrom,id=drive-ide0-1-0
-device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
-device virtio-net-pci,vlan=0,id=net0,mac=52:54:00:24:a5:1d,bus=pci.0,addr=0x5
-net tap,fd=15,vlan=0,name=hostnet0 -usb -vnc *:0 -k en-us -vga cirrus
-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3
19:22:40.436: debug : qemuSecurityDACSetProcessLabel:411 : Dropping
privileges of VM to 0:0
backtrace() returned 16 addresses
/usr/bin/qemu-kvm(bye+0x2b) [0x43346e]
/lib64/libc.so.6(+0x355e5) [0x7f45c50605e5]
/lib64/libc.so.6(+0x35635) [0x7f45c5060635]
/usr/bin/qemu-kvm() [0x4489e2]
/usr/bin/qemu-kvm(virtio_queue_notify+0x89) [0x5bb841]
/usr/bin/qemu-kvm() [0x449bb3]
/usr/bin/qemu-kvm() [0x44a077]
/usr/bin/qemu-kvm() [0x4f2cb4]
/usr/bin/qemu-kvm(cpu_outw+0x20) [0x4f305d]
/usr/bin/qemu-kvm() [0x4513f8]
/usr/bin/qemu-kvm(kvm_run+0x2e0) [0x4539bb]
/usr/bin/qemu-kvm(kvm_cpu_exec+0x15) [0x454c8c]
/usr/bin/qemu-kvm() [0x45541f]
/usr/bin/qemu-kvm() [0x455571]
/lib64/libpthread.so.0(+0x75f0) [0x7f45c75905f0]
/lib64/libc.so.6(clone+0x6d) [0x7f45c50ff84d]

my question :
for vcpu thread, why block all signal except SIGBUS SIGIPI? is it safe
to do register another signal?

Re: collect some information when qemu-kvm exit

[lidong chen]

Hi all,

I didn't understand why need block all signal except SIGBUS SIGIPI for
vcpu thread? is it safe to register another signal handler?
if somebody know the reason, please tell me.

and is it worth to do this?

2011/6/22 lidong chen <chen.lidong.kernel@gmail.com>:
> I find qemu-kvm only output a little information when abnormally exit.
> For example, if qemu-kvm exit by segmentation fault, there are no
> information in /var/log/libvirt/qemu/xx.log.
>
> so i want to solve this by collect some information when qemu-kvm exit.
>
> my idea is register some signal handler, and print some debug
> information in the signal handler function. and use atexit register a
> function for process termination.
>
> the main thread already used SIGBUS,SIGUSR2,SIGALRM,SIGIO. and  vcpu
> thread used SIGBUS,SIGIPI.
>
> the signal that should register for vcpu thread.
>    SIGHUP,
>    SIGINT,
>    SIGQUIT,
>    SIGILL,
>    SIGTRAP,
>    SIGABRT,
>    SIGFPE,
>    SIGUSR1,
>    SIGPIPE,
>    SIGSEGV,
>    SIGUSR2,
>    SIGTERM,
>    SIGSTKFLT,
>    SIGTSTP,
>    SIGXCPU,
>    SIGXFSZ,
>    SIGVTALRM,
>    SIGIO,
>    SIGSYS,
>
> the signal that should register for main thread.
>    SIGHUP,
>    SIGINT,
>    SIGQUIT,
>    SIGILL,
>    SIGTRAP,
>    SIGABRT,
>    SIGFPE,
>    SIGUSR1,
>    SIGPIPE,
>    SIGSEGV,
>    SIGTERM,
>    SIGSTKFLT,
>    SIGTSTP,
>    SIGXCPU,
>    SIGXFSZ,
>    SIGVTALRM,
>    SIGSYS,
>
> the simple example for this function:
> --- ../../BUILD/qemu-kvm-0.12.5/vl.c 2011-05-25 04:08:00.000000000 -0400
> +++ vl.c 2011-06-22 06:57:51.000000000 -0400
> +static void sig_handler(int n)
> +{
> +         int j, nptrs;
> +       #define SIZE 100
> +           void *buffer[100];
> +           char **strings;
> +
> +           nptrs = backtrace(buffer, SIZE);
> +           printf("backtrace() returned %d addresses\n", nptrs);
> +
> +           /* The call backtrace_symbols_fd(buffer, nptrs, STDOUT_FILENO)
> +              would produce similar output to the following: */
> +
> +           strings = backtrace_symbols(buffer, nptrs);
> +           if (strings == NULL) {
> +               perror("backtrace_symbols");
> +               exit(EXIT_FAILURE);
> +           }
> +
> +           for (j = 0; j < nptrs; j++)
> +               printf("%s\n", strings[j]);
> +
> +           free(strings);
> +
> +       abort();
> +}
> +
> +void bye(void) {
> +        printf("That was all, folks\n");
> +         int j, nptrs;
> +       #define SIZE 100
> +           void *buffer[100];
> +           char **strings;
> +
> +           nptrs = backtrace(buffer, SIZE);
> +           printf("backtrace() returned %d addresses\n", nptrs);
> +
> +           /* The call backtrace_symbols_fd(buffer, nptrs, STDOUT_FILENO)
> +              would produce similar output to the following: */
> +
> +           strings = backtrace_symbols(buffer, nptrs);
> +           if (strings == NULL) {
> +               perror("backtrace_symbols");
> +               exit(EXIT_FAILURE);
> +           }
> +
> +           for (j = 0; j < nptrs; j++)
> +               printf("%s\n", strings[j]);
> +
> +           free(strings);
> +
> +}
> +
> +
> int main(int argc, char **argv, char **envp)
> {
>     const char *gdbstub_dev = NULL;
> @@ -4954,6 +5010,17 @@
>
>     init_clocks();
>
> +
> +    signal(SIGSEGV, sig_handler);
> +
> +    i = atexit(bye);
> +    if (i != 0) {
> +           fprintf(stderr, "cannot set exit function\n");
> +           return EXIT_FAILURE;
> +    }
> +
> +
> +
>
> --- ../../BUILD/qemu-kvm-0.12.5/qemu-kvm.c    2011-05-25
> 03:31:25.000000000 -0400
> +++ qemu-kvm.c    2011-06-22 07:04:05.000000000 -0400
> @@ -1883,6 +1883,10 @@
>
> +
> static void *ap_main_loop(void *_env)
> {
>     CPUState *env = _env;
> @@ -1908,10 +1927,10 @@
>     struct ioperm_data *data = NULL;
> #endif
>
>     current_env = env;
>     env->thread_id = kvm_get_thread_id();
>     sigfillset(&signals);
> +    sigdelset(&signals,SIGSEGV);
>     sigprocmask(SIG_BLOCK, &signals, NULL);
>     kvm_create_vcpu(env, env->cpu_index);
>
> the log in /var/log/libvirt/qemu/xx.log.
>
> LC_ALL=C PATH=/bin:/sbin:/usr/bin:/usr/sbin HOME=/ QEMU_AUDIO_DRV=none
> /usr/bin/qemu-kvm -S -M pc-0.12 -enable-kvm -m 9767 -smp
> 16,sockets=16,cores=1,threads=1 -name sles11-4 -uuid
> 52841129-6b46-fef5-6a62-11c422597206 -nodefaults -chardev
> socket,id=monitor,path=/var/lib/libvirt/qemu/sles11-4.monitor,server,nowait
> -mon chardev=monitor,mode=readline -no-reboot -boot dc -drive
> file=/mnt/disk2/c00104598/sles11sp1.raw,if=none,id=drive-virtio-disk0,boot=on
> -device virtio-blk-pci,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0
> -drive file=/opt/c00104598/iso/SLES-11-SP1.iso,if=none,media=cdrom,id=drive-ide0-1-0
> -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
> -device virtio-net-pci,vlan=0,id=net0,mac=52:54:00:24:a5:1d,bus=pci.0,addr=0x5
> -net tap,fd=15,vlan=0,name=hostnet0 -usb -vnc *:0 -k en-us -vga cirrus
> -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x3
> 19:22:40.436: debug : qemuSecurityDACSetProcessLabel:411 : Dropping
> privileges of VM to 0:0
> backtrace() returned 16 addresses
> /usr/bin/qemu-kvm(bye+0x2b) [0x43346e]
> /lib64/libc.so.6(+0x355e5) [0x7f45c50605e5]
> /lib64/libc.so.6(+0x35635) [0x7f45c5060635]
> /usr/bin/qemu-kvm() [0x4489e2]
> /usr/bin/qemu-kvm(virtio_queue_notify+0x89) [0x5bb841]
> /usr/bin/qemu-kvm() [0x449bb3]
> /usr/bin/qemu-kvm() [0x44a077]
> /usr/bin/qemu-kvm() [0x4f2cb4]
> /usr/bin/qemu-kvm(cpu_outw+0x20) [0x4f305d]
> /usr/bin/qemu-kvm() [0x4513f8]
> /usr/bin/qemu-kvm(kvm_run+0x2e0) [0x4539bb]
> /usr/bin/qemu-kvm(kvm_cpu_exec+0x15) [0x454c8c]
> /usr/bin/qemu-kvm() [0x45541f]
> /usr/bin/qemu-kvm() [0x455571]
> /lib64/libpthread.so.0(+0x75f0) [0x7f45c75905f0]
> /lib64/libc.so.6(clone+0x6d) [0x7f45c50ff84d]
>
> my question :
> for vcpu thread, why block all signal except SIGBUS SIGIPI? is it safe
> to do register another signal?
>

Re: collect some information when qemu-kvm exit

[Jan Kiszka]

[-- Attachment #1: Type: text/plain, Size: 660 bytes --]

On 2011-06-23 11:05, lidong chen wrote:
> Hi all,
> 
> I didn't understand why need block all signal except SIGBUS SIGIPI for
> vcpu thread?

For simplicity reasons: All other expected signals are handled by the
io-thread or other helper threads. Those must never be processed by the
vcpus. So we block everything except the required ones.

> is it safe to register another signal handler?
> if somebody know the reason, please tell me.
> 
> and is it worth to do this?

Better configure your host to create a core dump of the dying process.
That contains more information than you could ever collect ad-hoc by
patching qemu itself.

Jan


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 259 bytes --]

Re: collect some information when qemu-kvm exit

[lidong chen]

2011/6/23 Jan Kiszka <jan.kiszka@web.de>:
> On 2011-06-23 11:05, lidong chen wrote:
>> Hi all,
>>
>> I didn't understand why need block all signal except SIGBUS SIGIPI for
>> vcpu thread?
>
> For simplicity reasons: All other expected signals are handled by the
> io-thread or other helper threads. Those must never be processed by the
> vcpus. So we block everything except the required ones.
>

the signals like SIGSEGV,SIGTRAP, didn't used for io-thread. and if
vcpu thread cause
segmentation fault, it's better handle by vcpu thread itself.


>> is it safe to register another signal handler?
>> if somebody know the reason, please tell me.
>>
>> and is it worth to do this?
>
because the core dump file is too big, and the time of core dump is too long.
I do a test, for a guest which have 9.7G memory, the coredump file is
9.7G, and the time of core dump is 1 minute.

for the compute node in my system, there are a lot of  cpu and memory
resource, but no disk.


total 4.5G
-rw------- 1 root root 9.7G Jun 23 21:31 core-qemu-kvm-24090-1308835893
-rw------- 1 root root 3.9G Jun 23 21:34 core-qemu-kvm-24098-1308835996

> Better configure your host to create a core dump of the dying process.
> That contains more information than you could ever collect ad-hoc by
> patching qemu itself.
>
> Jan
>
>

Re: collect some information when qemu-kvm exit

[Jan Kiszka]

[-- Attachment #1: Type: text/plain, Size: 1000 bytes --]

On 2011-06-23 15:56, lidong chen wrote:
>>> is it safe to register another signal handler?
>>> if somebody know the reason, please tell me.
>>>
>>> and is it worth to do this?
>>
> because the core dump file is too big, and the time of core dump is too long.
> I do a test, for a guest which have 9.7G memory, the coredump file is
> 9.7G, and the time of core dump is 1 minute.
> 
> for the compute node in my system, there are a lot of  cpu and memory
> resource, but no disk.
> 
> 
> total 4.5G
> -rw------- 1 root root 9.7G Jun 23 21:31 core-qemu-kvm-24090-1308835893
> -rw------- 1 root root 3.9G Jun 23 21:34 core-qemu-kvm-24098-1308835996

ulimit -c allows you to restrict the core file size so that it fits on
your ram disk. That will at least collect enough information to do a
proper post-mortem backtrace in gdb, including register states. It also
allows to inspect variables on the stacks and the heap. No need to add a
singe line or code to qemu for this.

Jan


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 259 bytes --]

Re: collect some information when qemu-kvm exit

[lidong chen]

2011/6/23 Jan Kiszka <jan.kiszka@web.de>:
> On 2011-06-23 15:56, lidong chen wrote:
>>>> is it safe to register another signal handler?
>>>> if somebody know the reason, please tell me.
>>>>
>>>> and is it worth to do this?
>>>
>> because the core dump file is too big, and the time of core dump is too long.
>> I do a test, for a guest which have 9.7G memory, the coredump file is
>> 9.7G, and the time of core dump is 1 minute.
>>
>> for the compute node in my system, there are a lot of  cpu and memory
>> resource, but no disk.
>>
>>
>> total 4.5G
>> -rw------- 1 root root 9.7G Jun 23 21:31 core-qemu-kvm-24090-1308835893
>> -rw------- 1 root root 3.9G Jun 23 21:34 core-qemu-kvm-24098-1308835996
>
> ulimit -c allows you to restrict the core file size so that it fits on
> your ram disk. That will at least collect enough information to do a
> proper post-mortem backtrace in gdb, including register states. It also
> allows to inspect variables on the stacks and the heap. No need to add a
> singe line or code to qemu for this.
>

if i use 'ulimit -c 6000' to restrict the core file, the backtrace
can't work correctly.

26:/corefile # gdb /usr/bin/qemu-kvm core-qemu-kvm-9979-1308888098
GNU gdb (GDB) SUSE (7.0-0.4.16)
Copyright (C) 2009 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-suse-linux".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/bin/qemu-kvm...done.
BFD: Warning: /corefile/core-qemu-kvm-9979-1308888098 is truncated:
expected core file size >= 10583695360, found: 10051584.
[New Thread 9981]
[New Thread 9980]
Cannot access memory at address 0x7fb26cac1108
(gdb) backtrace
Cannot access memory at address 0x7fff489c03b0

if i use 'ulimit -c unlimited', it work correctly.
Core was generated by `qemu-kvm -hda
/var/lib/libvirt/images/sles11-4.img -boot dc -m 9999 -vnc *:0 -k'.
Program terminated with signal 11, Segmentation fault.
#0  0x00007f6ba768cb93 in select () from /lib64/libc.so.6
(gdb) bt
#0  0x00007f6ba768cb93 in select () from /lib64/libc.so.6
#1  0x0000000000431b8a in main_loop_wait (timeout=1000)
    at /usr/src/packages/BUILD_chen/qemu-kvm-0.12.5/vl.c:4006
#2  0x0000000000455a50 in kvm_main_loop ()
    at /usr/src/packages/BUILD_chen/qemu-kvm-0.12.5/qemu-kvm.c:2129
#3  0x00000000004322bc in main_loop () at
/usr/src/packages/BUILD_chen/qemu-kvm-0.12.5/vl.c:4231
#4  0x0000000000435fb3 in main (argc=11, argv=0x7fffadc777d8,
envp=0x7fffadc77838)
    at /usr/src/packages/BUILD_chen/qemu-kvm-0.12.5/vl.c:6356
(gdb)

I google how could gdb handle truncated core files,
http://sourceware.org/ml/gdb/2008-08/msg00276.html

there are two potential way to support gdb handle truncated core files:
1.modify the code of linux kernel, to support for prioritisation of
what is dumped.
2.used coredumper,  http://code.google.com/p/google-coredumper/

i read the code of function elf_core_dump in linux kernel, it write
each vma in sequence.

> Jan
>
>

Re: collect some information when qemu-kvm exit

[Jan Kiszka]

On 2011-06-24 10:24, lidong chen wrote:
> 2011/6/23 Jan Kiszka <jan.kiszka@web.de>:
>> On 2011-06-23 15:56, lidong chen wrote:
>>>>> is it safe to register another signal handler?
>>>>> if somebody know the reason, please tell me.
>>>>>
>>>>> and is it worth to do this?
>>>>
>>> because the core dump file is too big, and the time of core dump is too long.
>>> I do a test, for a guest which have 9.7G memory, the coredump file is
>>> 9.7G, and the time of core dump is 1 minute.
>>>
>>> for the compute node in my system, there are a lot of  cpu and memory
>>> resource, but no disk.
>>>
>>>
>>> total 4.5G
>>> -rw------- 1 root root 9.7G Jun 23 21:31 core-qemu-kvm-24090-1308835893
>>> -rw------- 1 root root 3.9G Jun 23 21:34 core-qemu-kvm-24098-1308835996
>>
>> ulimit -c allows you to restrict the core file size so that it fits on
>> your ram disk. That will at least collect enough information to do a
>> proper post-mortem backtrace in gdb, including register states. It also
>> allows to inspect variables on the stacks and the heap. No need to add a
>> singe line or code to qemu for this.
>>
> 
> if i use 'ulimit -c 6000' to restrict the core file, the backtrace
> can't work correctly.

I've granted a few hundred megs, and it worked for me.

> 
> 26:/corefile # gdb /usr/bin/qemu-kvm core-qemu-kvm-9979-1308888098
> GNU gdb (GDB) SUSE (7.0-0.4.16)
> Copyright (C) 2009 Free Software Foundation, Inc.
> License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
> This is free software: you are free to change and redistribute it.
> There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
> and "show warranty" for details.
> This GDB was configured as "x86_64-suse-linux".
> For bug reporting instructions, please see:
> <http://www.gnu.org/software/gdb/bugs/>...
> Reading symbols from /usr/bin/qemu-kvm...done.
> BFD: Warning: /corefile/core-qemu-kvm-9979-1308888098 is truncated:
> expected core file size >= 10583695360, found: 10051584.
> [New Thread 9981]
> [New Thread 9980]
> Cannot access memory at address 0x7fb26cac1108
> (gdb) backtrace
> Cannot access memory at address 0x7fff489c03b0
> 
> if i use 'ulimit -c unlimited', it work correctly.
> Core was generated by `qemu-kvm -hda
> /var/lib/libvirt/images/sles11-4.img -boot dc -m 9999 -vnc *:0 -k'.
> Program terminated with signal 11, Segmentation fault.
> #0  0x00007f6ba768cb93 in select () from /lib64/libc.so.6
> (gdb) bt
> #0  0x00007f6ba768cb93 in select () from /lib64/libc.so.6
> #1  0x0000000000431b8a in main_loop_wait (timeout=1000)
>     at /usr/src/packages/BUILD_chen/qemu-kvm-0.12.5/vl.c:4006
> #2  0x0000000000455a50 in kvm_main_loop ()
>     at /usr/src/packages/BUILD_chen/qemu-kvm-0.12.5/qemu-kvm.c:2129
> #3  0x00000000004322bc in main_loop () at
> /usr/src/packages/BUILD_chen/qemu-kvm-0.12.5/vl.c:4231
> #4  0x0000000000435fb3 in main (argc=11, argv=0x7fffadc777d8,
> envp=0x7fffadc77838)
>     at /usr/src/packages/BUILD_chen/qemu-kvm-0.12.5/vl.c:6356
> (gdb)
> 
> I google how could gdb handle truncated core files,
> http://sourceware.org/ml/gdb/2008-08/msg00276.html
> 
> there are two potential way to support gdb handle truncated core files:
> 1.modify the code of linux kernel, to support for prioritisation of
> what is dumped.

What we basically need to avoid playing with ulimit is a way to tell the
kernel "do not dump this region", and apply that on the guest memory at
least.

> 2.used coredumper,  http://code.google.com/p/google-coredumper/
> 
> i read the code of function elf_core_dump in linux kernel, it write
> each vma in sequence.

...optionally prioritized by size - or that way. Though we wouldn't need
it given a proper kernel interface. Don't know why such thing does not
exist yet.

Jan

-- 
Siemens AG, Corporate Technology, CT T DE IT 1
Corporate Competence Center Embedded Linux

Re: collect some information when qemu-kvm exit

[Jan Kiszka]

On 2011-06-24 10:55, Jan Kiszka wrote:
> On 2011-06-24 10:24, lidong chen wrote:
>> 2011/6/23 Jan Kiszka <jan.kiszka@web.de>:
>>> On 2011-06-23 15:56, lidong chen wrote:
>>>>>> is it safe to register another signal handler?
>>>>>> if somebody know the reason, please tell me.
>>>>>>
>>>>>> and is it worth to do this?
>>>>>
>>>> because the core dump file is too big, and the time of core dump is too long.
>>>> I do a test, for a guest which have 9.7G memory, the coredump file is
>>>> 9.7G, and the time of core dump is 1 minute.
>>>>
>>>> for the compute node in my system, there are a lot of  cpu and memory
>>>> resource, but no disk.
>>>>
>>>>
>>>> total 4.5G
>>>> -rw------- 1 root root 9.7G Jun 23 21:31 core-qemu-kvm-24090-1308835893
>>>> -rw------- 1 root root 3.9G Jun 23 21:34 core-qemu-kvm-24098-1308835996
>>>
>>> ulimit -c allows you to restrict the core file size so that it fits on
>>> your ram disk. That will at least collect enough information to do a
>>> proper post-mortem backtrace in gdb, including register states. It also
>>> allows to inspect variables on the stacks and the heap. No need to add a
>>> singe line or code to qemu for this.
>>>
>>
>> if i use 'ulimit -c 6000' to restrict the core file, the backtrace
>> can't work correctly.
> 
> I've granted a few hundred megs, and it worked for me.
> 
>>
>> 26:/corefile # gdb /usr/bin/qemu-kvm core-qemu-kvm-9979-1308888098
>> GNU gdb (GDB) SUSE (7.0-0.4.16)

And I've gdb 7.2.50.20101006-cvs here. Maybe that also contributes to a
working setup.

Jan

-- 
Siemens AG, Corporate Technology, CT T DE IT 1
Corporate Competence Center Embedded Linux

Re: collect some information when qemu-kvm exit

[lidong chen]

2011/6/24 Jan Kiszka <jan.kiszka@siemens.com>:
> On 2011-06-24 10:55, Jan Kiszka wrote:
>> On 2011-06-24 10:24, lidong chen wrote:
>>> 2011/6/23 Jan Kiszka <jan.kiszka@web.de>:
>>>> On 2011-06-23 15:56, lidong chen wrote:
>>>>>>> is it safe to register another signal handler?
>>>>>>> if somebody know the reason, please tell me.
>>>>>>>
>>>>>>> and is it worth to do this?
>>>>>>
>>>>> because the core dump file is too big, and the time of core dump is too long.
>>>>> I do a test, for a guest which have 9.7G memory, the coredump file is
>>>>> 9.7G, and the time of core dump is 1 minute.
>>>>>
>>>>> for the compute node in my system, there are a lot of  cpu and memory
>>>>> resource, but no disk.
>>>>>
>>>>>
>>>>> total 4.5G
>>>>> -rw------- 1 root root 9.7G Jun 23 21:31 core-qemu-kvm-24090-1308835893
>>>>> -rw------- 1 root root 3.9G Jun 23 21:34 core-qemu-kvm-24098-1308835996
>>>>
>>>> ulimit -c allows you to restrict the core file size so that it fits on
>>>> your ram disk. That will at least collect enough information to do a
>>>> proper post-mortem backtrace in gdb, including register states. It also
>>>> allows to inspect variables on the stacks and the heap. No need to add a
>>>> singe line or code to qemu for this.
>>>>
>>>
>>> if i use 'ulimit -c 6000' to restrict the core file, the backtrace
>>> can't work correctly.
>>
>> I've granted a few hundred megs, and it worked for me.
>>
>>>
>>> 26:/corefile # gdb /usr/bin/qemu-kvm core-qemu-kvm-9979-1308888098
>>> GNU gdb (GDB) SUSE (7.0-0.4.16)
>
> And I've gdb 7.2.50.20101006-cvs here. Maybe that also contributes to a
> working setup.
>

i try newer version of gdb, it still can't work correctly.

26:/corefile # /usr/local/bin/gdb /usr/bin/qemu-kvm
core-qemu-kvm-20458-1308912006
GNU gdb (GDB) 7.2.90.20110530-cvs
Copyright (C) 2011 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-unknown-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/bin/qemu-kvm...done.
[New LWP 20458]
[New LWP 20459]
Cannot access memory at address 0x7fbffdca3108
Core was generated by `qemu-kvm -hda
/var/lib/libvirt/images/sles11-4.img -boot dc -m 9999 -vnc *:0 -k'.
Program terminated with signal 11, Segmentation fault.
#0  0x00007fbffb1ccb93 in ?? ()
(gdb) bt
#0  0x00007fbffb1ccb93 in ?? ()
Cannot access memory at address 0x7ffffc6daaf0

to tell kernel didn't dump guest os memory, i set coredump_filter to
0x32, cancel the dump of anonymous private mappings.
the coredump_filter is 0x33 default,

echo 0x32 > /proc/14528/coredump_filter

and the size of core file is only 224K.
-rw------- 1 root root 224K Jun 24 20:46 core-qemu-kvm-14528-1308919614

but the gdb can't not work correctly.
GNU gdb (GDB) 7.2.90.20110530-cvs
Copyright (C) 2011 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-unknown-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/bin/qemu-kvm...done.
[New LWP 14528]
[New LWP 14529]
[New LWP 14537]
Core was generated by `qemu-kvm -hda
/var/lib/libvirt/images/sles11-4.img -boot dc -m 9999 -vnc *:0 -k'.
Program terminated with signal 11, Segmentation fault.
#0  0x00007f8988221b93 in ?? ()
(gdb) bt
#0  0x00007f8988221b93 in ?? ()
Cannot access memory at address 0x7fffec55dee0
(gdb)

i guess the address 0x00007f8988221b93 is in select from /lib64/libc.so.6.



> Jan
>
> --
> Siemens AG, Corporate Technology, CT T DE IT 1
> Corporate Competence Center Embedded Linux
>

Re: collect some information when qemu-kvm exit

[lidong chen]

I think there are no way to tell the kernel didn't dump guest os memory.
for current kernel, /proc/<pid>/coredump_filter only support the
following 7 memory types.

  - (bit 0) anonymous private memory
  - (bit 1) anonymous shared memory
  - (bit 2) file-backed private memory
  - (bit 3) file-backed shared memory
  - (bit 4) ELF header pages in file-backed private memory areas (it
is effective only if the bit 2 is cleared)
  - (bit 5) hugetlb private memory
  - (bit 6) hugetlb shared memory

but for a process, there are also other anonymous private memory vma.

i modify the funcation vma_dump_size like this, only didn't dump guest
memory. then gdb can work correctly and the core file is 22M.

maybe we need add a option for coredump_filter.

--- /usr/src/linux/fs/binfmt_elf.c	2011-05-10 08:02:45.000000000 -0400
+++ fs/binfmt_elf.c	2011-07-02 06:30:17.000000000 -0400
@@ -1154,8 +1154,11 @@
 	}

 	/* Dump segments that have been written to.  */
-	if (vma->anon_vma && FILTER(ANON_PRIVATE))
+	if (vma->anon_vma && !(vma->vm_flags & VM_MERGEABLE) ) {
 		goto whole;
+        } else if ( vma->anon_vma && FILTER(ANON_PRIVATE) ) {
+                goto whole;
+        }
 	if (vma->vm_file == NULL)
 		return 0;

linux-gzvm:/opt/c00104598/coretest # gdb main core
GNU gdb (GDB) SUSE (7.0-0.4.16)
Copyright (C) 2009 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-suse-linux".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /opt/c00104598/coretest/main...done.

warning: core file may not match specified executable file.
[New Thread 9099]
Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2
Try: zypper install -C
"debuginfo(build-id)=17c088070352d83e7afc43d83756b00899fd37f0"
Core was generated by `/usr/bin/qemu-kvm -S -M pc-0.12 -enable-kvm -m
512 -nodefaults -boot c -hda /op'.
Program terminated with signal 11, Segmentation fault.
#0  0x000000000049a577 in ?? ()
(gdb) bt
#0  0x000000000049a577 in ?? ()
#1  0x00000000004d2385 in ?? ()
#2  0x0000000000f18960 in ?? ()
#3  0x0000000000f18960 in ?? ()
#4  0x00007fffa2a5ff40 in ?? ()
#5  0x00007fffa2a5ffc0 in ?? ()
#6  0x0000000000f18960 in ?? ()
#7  0x0000000000000014 in ?? ()
#8  0x00007fffa2a5ff40 in ?? ()
#9  0x00007fffa2a5ffc0 in ?? ()
#10 0x00007fffa2a60040 in ?? ()
#11 0x00000000004d46b1 in ?? ()
#12 0x00007fffa2a5ffc0 in ?? ()
#13 0x0000000000cb1d80 in ?? ()
#14 0x0000000000000001 in ?? ()
#15 0x000000000040c752 in ?? ()
#16 0x00007fac8c4e4128 in _r_debug ()
#17 0x000003e88c2cd933 in ?? ()
#18 0x0000000000000000 in ?? ()
(gdb)

2011/6/24 lidong chen <chen.lidong.kernel@gmail.com>:
> 2011/6/24 Jan Kiszka <jan.kiszka@siemens.com>:
>> On 2011-06-24 10:55, Jan Kiszka wrote:
>>> On 2011-06-24 10:24, lidong chen wrote:
>>>> 2011/6/23 Jan Kiszka <jan.kiszka@web.de>:
>>>>> On 2011-06-23 15:56, lidong chen wrote:
>>>>>>>> is it safe to register another signal handler?
>>>>>>>> if somebody know the reason, please tell me.
>>>>>>>>
>>>>>>>> and is it worth to do this?
>>>>>>>
>>>>>> because the core dump file is too big, and the time of core dump is too long.
>>>>>> I do a test, for a guest which have 9.7G memory, the coredump file is
>>>>>> 9.7G, and the time of core dump is 1 minute.
>>>>>>
>>>>>> for the compute node in my system, there are a lot of  cpu and memory
>>>>>> resource, but no disk.
>>>>>>
>>>>>>
>>>>>> total 4.5G
>>>>>> -rw------- 1 root root 9.7G Jun 23 21:31 core-qemu-kvm-24090-1308835893
>>>>>> -rw------- 1 root root 3.9G Jun 23 21:34 core-qemu-kvm-24098-1308835996
>>>>>
>>>>> ulimit -c allows you to restrict the core file size so that it fits on
>>>>> your ram disk. That will at least collect enough information to do a
>>>>> proper post-mortem backtrace in gdb, including register states. It also
>>>>> allows to inspect variables on the stacks and the heap. No need to add a
>>>>> singe line or code to qemu for this.
>>>>>
>>>>
>>>> if i use 'ulimit -c 6000' to restrict the core file, the backtrace
>>>> can't work correctly.
>>>
>>> I've granted a few hundred megs, and it worked for me.
>>>
>>>>
>>>> 26:/corefile # gdb /usr/bin/qemu-kvm core-qemu-kvm-9979-1308888098
>>>> GNU gdb (GDB) SUSE (7.0-0.4.16)
>>
>> And I've gdb 7.2.50.20101006-cvs here. Maybe that also contributes to a
>> working setup.
>>
>
> i try newer version of gdb, it still can't work correctly.
>
> 26:/corefile # /usr/local/bin/gdb /usr/bin/qemu-kvm
> core-qemu-kvm-20458-1308912006
> GNU gdb (GDB) 7.2.90.20110530-cvs
> Copyright (C) 2011 Free Software Foundation, Inc.
> License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
> This is free software: you are free to change and redistribute it.
> There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
> and "show warranty" for details.
> This GDB was configured as "x86_64-unknown-linux-gnu".
> For bug reporting instructions, please see:
> <http://www.gnu.org/software/gdb/bugs/>...
> Reading symbols from /usr/bin/qemu-kvm...done.
> [New LWP 20458]
> [New LWP 20459]
> Cannot access memory at address 0x7fbffdca3108
> Core was generated by `qemu-kvm -hda
> /var/lib/libvirt/images/sles11-4.img -boot dc -m 9999 -vnc *:0 -k'.
> Program terminated with signal 11, Segmentation fault.
> #0  0x00007fbffb1ccb93 in ?? ()
> (gdb) bt
> #0  0x00007fbffb1ccb93 in ?? ()
> Cannot access memory at address 0x7ffffc6daaf0
>
> to tell kernel didn't dump guest os memory, i set coredump_filter to
> 0x32, cancel the dump of anonymous private mappings.
> the coredump_filter is 0x33 default,
>
> echo 0x32 > /proc/14528/coredump_filter
>
> and the size of core file is only 224K.
> -rw------- 1 root root 224K Jun 24 20:46 core-qemu-kvm-14528-1308919614
>
> but the gdb can't not work correctly.
> GNU gdb (GDB) 7.2.90.20110530-cvs
> Copyright (C) 2011 Free Software Foundation, Inc.
> License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
> This is free software: you are free to change and redistribute it.
> There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
> and "show warranty" for details.
> This GDB was configured as "x86_64-unknown-linux-gnu".
> For bug reporting instructions, please see:
> <http://www.gnu.org/software/gdb/bugs/>...
> Reading symbols from /usr/bin/qemu-kvm...done.
> [New LWP 14528]
> [New LWP 14529]
> [New LWP 14537]
> Core was generated by `qemu-kvm -hda
> /var/lib/libvirt/images/sles11-4.img -boot dc -m 9999 -vnc *:0 -k'.
> Program terminated with signal 11, Segmentation fault.
> #0  0x00007f8988221b93 in ?? ()
> (gdb) bt
> #0  0x00007f8988221b93 in ?? ()
> Cannot access memory at address 0x7fffec55dee0
> (gdb)
>
> i guess the address 0x00007f8988221b93 is in select from /lib64/libc.so.6.
>
>
>
>> Jan
>>
>> --
>> Siemens AG, Corporate Technology, CT T DE IT 1
>> Corporate Competence Center Embedded Linux
>>
>

Re: collect some information when qemu-kvm exit

[lidong chen]

Hi Kawai:

I find that if set coredump_filter to 0x32, the core file can't gdb
work correctly.
because the value of anon_vma for stack or brk vma is also not NULL.

in function vma_dump_size, stack and brk vma will not dump if
FILTER(ANON_PRIVATE) is 0.
if ( vma->anon_vma && FILTER(ANON_PRIVATE) ) {
      goto whole;
 }

are there any method only didn't dump anonymous private vma caused by mmap?

the purpose i do this is to support qemu-kvm didn't dump guest os
memory, which is  anonymous private memory caused by mmap system call.

i find another way to implement this function, but i think is not the best way.
because the guest os memory also have VM_MERGEABLE flag, so i can add
a option in
coredump_filter and only didn't dump guest os memory.

        /* Dump segments that have been written to.  */
 -       if (vma->anon_vma && FILTER(ANON_PRIVATE))
 +       if (vma->anon_vma && !(vma->vm_flags & VM_MERGEABLE) ) {
                goto whole;
 +        } else if ( vma->anon_vma && FILTER(ANON_PRIVATE) ) {
 +                goto whole;
 +        }
        if (vma->vm_file == NULL)
                return 0;

thanks.

2011/7/2 lidong chen <chen.lidong.kernel@gmail.com>:
> I think there are no way to tell the kernel didn't dump guest os memory.
> for current kernel, /proc/<pid>/coredump_filter only support the
> following 7 memory types.
>
>  - (bit 0) anonymous private memory
>  - (bit 1) anonymous shared memory
>  - (bit 2) file-backed private memory
>  - (bit 3) file-backed shared memory
>  - (bit 4) ELF header pages in file-backed private memory areas (it
> is effective only if the bit 2 is cleared)
>  - (bit 5) hugetlb private memory
>  - (bit 6) hugetlb shared memory
>
> but for a process, there are also other anonymous private memory vma.
>
> i modify the funcation vma_dump_size like this, only didn't dump guest
> memory. then gdb can work correctly and the core file is 22M.
>
> maybe we need add a option for coredump_filter.
>
> --- /usr/src/linux/fs/binfmt_elf.c      2011-05-10 08:02:45.000000000 -0400
> +++ fs/binfmt_elf.c     2011-07-02 06:30:17.000000000 -0400
> @@ -1154,8 +1154,11 @@
>        }
>
>        /* Dump segments that have been written to.  */
> -       if (vma->anon_vma && FILTER(ANON_PRIVATE))
> +       if (vma->anon_vma && !(vma->vm_flags & VM_MERGEABLE) ) {
>                goto whole;
> +        } else if ( vma->anon_vma && FILTER(ANON_PRIVATE) ) {
> +                goto whole;
> +        }
>        if (vma->vm_file == NULL)
>                return 0;
>
> linux-gzvm:/opt/c00104598/coretest # gdb main core
> GNU gdb (GDB) SUSE (7.0-0.4.16)
> Copyright (C) 2009 Free Software Foundation, Inc.
> License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
> This is free software: you are free to change and redistribute it.
> There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
> and "show warranty" for details.
> This GDB was configured as "x86_64-suse-linux".
> For bug reporting instructions, please see:
> <http://www.gnu.org/software/gdb/bugs/>...
> Reading symbols from /opt/c00104598/coretest/main...done.
>
> warning: core file may not match specified executable file.
> [New Thread 9099]
> Missing separate debuginfo for /lib64/ld-linux-x86-64.so.2
> Try: zypper install -C
> "debuginfo(build-id)=17c088070352d83e7afc43d83756b00899fd37f0"
> Core was generated by `/usr/bin/qemu-kvm -S -M pc-0.12 -enable-kvm -m
> 512 -nodefaults -boot c -hda /op'.
> Program terminated with signal 11, Segmentation fault.
> #0  0x000000000049a577 in ?? ()
> (gdb) bt
> #0  0x000000000049a577 in ?? ()
> #1  0x00000000004d2385 in ?? ()
> #2  0x0000000000f18960 in ?? ()
> #3  0x0000000000f18960 in ?? ()
> #4  0x00007fffa2a5ff40 in ?? ()
> #5  0x00007fffa2a5ffc0 in ?? ()
> #6  0x0000000000f18960 in ?? ()
> #7  0x0000000000000014 in ?? ()
> #8  0x00007fffa2a5ff40 in ?? ()
> #9  0x00007fffa2a5ffc0 in ?? ()
> #10 0x00007fffa2a60040 in ?? ()
> #11 0x00000000004d46b1 in ?? ()
> #12 0x00007fffa2a5ffc0 in ?? ()
> #13 0x0000000000cb1d80 in ?? ()
> #14 0x0000000000000001 in ?? ()
> #15 0x000000000040c752 in ?? ()
> #16 0x00007fac8c4e4128 in _r_debug ()
> #17 0x000003e88c2cd933 in ?? ()
> #18 0x0000000000000000 in ?? ()
> (gdb)
>
> 2011/6/24 lidong chen <chen.lidong.kernel@gmail.com>:
>> 2011/6/24 Jan Kiszka <jan.kiszka@siemens.com>:
>>> On 2011-06-24 10:55, Jan Kiszka wrote:
>>>> On 2011-06-24 10:24, lidong chen wrote:
>>>>> 2011/6/23 Jan Kiszka <jan.kiszka@web.de>:
>>>>>> On 2011-06-23 15:56, lidong chen wrote:
>>>>>>>>> is it safe to register another signal handler?
>>>>>>>>> if somebody know the reason, please tell me.
>>>>>>>>>
>>>>>>>>> and is it worth to do this?
>>>>>>>>
>>>>>>> because the core dump file is too big, and the time of core dump is too long.
>>>>>>> I do a test, for a guest which have 9.7G memory, the coredump file is
>>>>>>> 9.7G, and the time of core dump is 1 minute.
>>>>>>>
>>>>>>> for the compute node in my system, there are a lot of  cpu and memory
>>>>>>> resource, but no disk.
>>>>>>>
>>>>>>>
>>>>>>> total 4.5G
>>>>>>> -rw------- 1 root root 9.7G Jun 23 21:31 core-qemu-kvm-24090-1308835893
>>>>>>> -rw------- 1 root root 3.9G Jun 23 21:34 core-qemu-kvm-24098-1308835996
>>>>>>
>>>>>> ulimit -c allows you to restrict the core file size so that it fits on
>>>>>> your ram disk. That will at least collect enough information to do a
>>>>>> proper post-mortem backtrace in gdb, including register states. It also
>>>>>> allows to inspect variables on the stacks and the heap. No need to add a
>>>>>> singe line or code to qemu for this.
>>>>>>
>>>>>
>>>>> if i use 'ulimit -c 6000' to restrict the core file, the backtrace
>>>>> can't work correctly.
>>>>
>>>> I've granted a few hundred megs, and it worked for me.
>>>>
>>>>>
>>>>> 26:/corefile # gdb /usr/bin/qemu-kvm core-qemu-kvm-9979-1308888098
>>>>> GNU gdb (GDB) SUSE (7.0-0.4.16)
>>>
>>> And I've gdb 7.2.50.20101006-cvs here. Maybe that also contributes to a
>>> working setup.
>>>
>>
>> i try newer version of gdb, it still can't work correctly.
>>
>> 26:/corefile # /usr/local/bin/gdb /usr/bin/qemu-kvm
>> core-qemu-kvm-20458-1308912006
>> GNU gdb (GDB) 7.2.90.20110530-cvs
>> Copyright (C) 2011 Free Software Foundation, Inc.
>> License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
>> This is free software: you are free to change and redistribute it.
>> There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
>> and "show warranty" for details.
>> This GDB was configured as "x86_64-unknown-linux-gnu".
>> For bug reporting instructions, please see:
>> <http://www.gnu.org/software/gdb/bugs/>...
>> Reading symbols from /usr/bin/qemu-kvm...done.
>> [New LWP 20458]
>> [New LWP 20459]
>> Cannot access memory at address 0x7fbffdca3108
>> Core was generated by `qemu-kvm -hda
>> /var/lib/libvirt/images/sles11-4.img -boot dc -m 9999 -vnc *:0 -k'.
>> Program terminated with signal 11, Segmentation fault.
>> #0  0x00007fbffb1ccb93 in ?? ()
>> (gdb) bt
>> #0  0x00007fbffb1ccb93 in ?? ()
>> Cannot access memory at address 0x7ffffc6daaf0
>>
>> to tell kernel didn't dump guest os memory, i set coredump_filter to
>> 0x32, cancel the dump of anonymous private mappings.
>> the coredump_filter is 0x33 default,
>>
>> echo 0x32 > /proc/14528/coredump_filter
>>
>> and the size of core file is only 224K.
>> -rw------- 1 root root 224K Jun 24 20:46 core-qemu-kvm-14528-1308919614
>>
>> but the gdb can't not work correctly.
>> GNU gdb (GDB) 7.2.90.20110530-cvs
>> Copyright (C) 2011 Free Software Foundation, Inc.
>> License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
>> This is free software: you are free to change and redistribute it.
>> There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
>> and "show warranty" for details.
>> This GDB was configured as "x86_64-unknown-linux-gnu".
>> For bug reporting instructions, please see:
>> <http://www.gnu.org/software/gdb/bugs/>...
>> Reading symbols from /usr/bin/qemu-kvm...done.
>> [New LWP 14528]
>> [New LWP 14529]
>> [New LWP 14537]
>> Core was generated by `qemu-kvm -hda
>> /var/lib/libvirt/images/sles11-4.img -boot dc -m 9999 -vnc *:0 -k'.
>> Program terminated with signal 11, Segmentation fault.
>> #0  0x00007f8988221b93 in ?? ()
>> (gdb) bt
>> #0  0x00007f8988221b93 in ?? ()
>> Cannot access memory at address 0x7fffec55dee0
>> (gdb)
>>
>> i guess the address 0x00007f8988221b93 is in select from /lib64/libc.so.6.
>>
>>
>>
>>> Jan
>>>
>>> --
>>> Siemens AG, Corporate Technology, CT T DE IT 1
>>> Corporate Competence Center Embedded Linux
>>>
>>
>